{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import argparse\n",
    "import json\n",
    "from collections import defaultdict, Counter\n",
    "import re\n",
    "from operator import itemgetter\n",
    "MAX_WORDS=40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we probably don't need the repeat dict ?\n",
    "# we don't need get_memory dict either \n",
    "# not handle_get_memroy either"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_prefix(text, prefix):\n",
    "    if text.startswith(prefix):\n",
    "        return text[len(prefix):]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert s to snake case\n",
    "def snake_case(s):\n",
    "    return re.sub(\"([a-z])([A-Z])\", \"\\\\1_\\\\2\", s).lower()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''this function splits the key that starts with a given prefix and only for values that are not None\n",
    "and makes the key be the thing after prefix\n",
    "'''\n",
    "def with_prefix(d, prefix):\n",
    "    new_d = {}\n",
    "    for k, v in d.items():\n",
    "        if k.startswith(prefix) and v not in (\"\", None, \"None\"):\n",
    "            index = k.find(prefix)+ len(prefix)\n",
    "            new_key = k[index:]\n",
    "            new_d[new_key] = v\n",
    "    return new_d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fix_nums_at_end_of_ref_obj(d, ps):    \n",
    "    for p in ps:\n",
    "        d = d.copy()\n",
    "        rm_keys = []\n",
    "        add_items = []\n",
    "        # print(p, d)\n",
    "        for k, v in d.items():\n",
    "            new_k = k.split(\".\")\n",
    "            updated_ks = []\n",
    "            #print(k, v, new_k, p)\n",
    "            for k1 in new_k:\n",
    "                #print(k1)\n",
    "                if k1.startswith(p) and k1[-1].isdigit():\n",
    "                    rm_keys.append(k)\n",
    "                    new_k1  = k1[:-1]\n",
    "                    updated_ks.append(new_k1)\n",
    "                    #add_items.append((k[:-1], v))\n",
    "                else:\n",
    "                    updated_ks.append(k1)\n",
    "            add_items.append((\".\".join(updated_ks), v))\n",
    "        for k in rm_keys:\n",
    "            del d[k]\n",
    "        for k, v in add_items:\n",
    "            d[k] = v\n",
    "    return d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reference_object.name_check.has_name.span#3': 'on',\n",
       " 'reference_object': 'name_check'}"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = {'reference_object.name_check.has_name.span#3': 'on', 'reference_object2': 'name_check'}\n",
    "\n",
    "fix_nums_at_end_of_ref_obj(x, ['reference_object'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' this function removes certain prefixes from keys and renames the key to be: key with text following \n",
    "the prefix in the dict'''\n",
    "def remove_key_prefixes(d, ps):\n",
    "    \n",
    "    for p in ps:\n",
    "        d = d.copy()\n",
    "        rm_keys = []\n",
    "        add_items = []\n",
    "        # print(p, d)\n",
    "        for k, v in d.items():\n",
    "            if k.startswith(p):\n",
    "                rm_keys.append(k)\n",
    "                add_items.append((k[len(p) :], v))\n",
    "        for k in rm_keys:\n",
    "            del d[k]\n",
    "        for k, v in add_items:\n",
    "            d[k] = v\n",
    "    return d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fix_spans_due_to_empty_words(action_dict, words):\n",
    "    \"\"\"Return modified (action_dict, words)\"\"\"\n",
    "\n",
    "    def reduce_span_vals_gte(d, i):\n",
    "        for k, v in d.items():\n",
    "            if type(v) == dict:\n",
    "                reduce_span_vals_gte(v, i)\n",
    "                continue\n",
    "            try:\n",
    "                a, b = v\n",
    "                if a >= i:\n",
    "                    a -= 1\n",
    "                if b >= i:\n",
    "                    b -= 1\n",
    "                d[k] = [[a, b]]\n",
    "            except ValueError:\n",
    "                pass\n",
    "            except TypeError:\n",
    "                pass\n",
    "\n",
    "    # remove trailing empty strings\n",
    "    while words[-1] == \"\":\n",
    "        del words[-1]\n",
    "\n",
    "    # fix span\n",
    "    i = 0\n",
    "    while i < len(words):\n",
    "        if words[i] == \"\":\n",
    "            reduce_span_vals_gte(action_dict, i)\n",
    "            del words[i]\n",
    "        else:\n",
    "            i += 1\n",
    "\n",
    "    return action_dict, words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' check for the following use cases:\\n1. plain ref object woith coref\\n2. ref obj with name, color etc\\n3. ref obj with name , color + filters\\n4. ref obj with name, filters and location\\n5. ref obj with filters only\\n'"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''' check for the following use cases:\n",
    "1. plain ref object woith coref\n",
    "2. ref obj with name, color etc\n",
    "3. ref obj with name , color + filters\n",
    "4. ref obj with name, filters and location\n",
    "5. ref obj with filters only\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_dict(d):\n",
    "    r = {}\n",
    "\n",
    "#     print(\"entererrr\")\n",
    "#     print(d)\n",
    "#     print(\"----------------\")\n",
    "    d = remove_key_prefixes(d, [\"name_check.\",\n",
    "                                \"size_check.\", \n",
    "                                'colour_check.', \n",
    "                                'block_type_check.', \n",
    "                                'arg_check.',\n",
    "                                'height_check.', \n",
    "                                'length_check.',\n",
    "                                'width_check.',\n",
    "                                'tag_check.',\n",
    "                                'thickness_check.',\n",
    "                                'coref_resolve_check.yes.',\n",
    "                                'coref_resolve_check.no.',\n",
    "                                'depth_check.',\n",
    "                               'reward.'])\n",
    "#     print(d)\n",
    "#     print(\"----------------new------------------\"\n",
    "    if \"location\" in d:\n",
    "#         print(d)\n",
    "        \n",
    "        # fix location type\n",
    "        r[\"location\"] = {\"location_type\": d[\"location\"]}\n",
    "        \n",
    "        # fix relative direction\n",
    "        reference_location_keys = [\"location.REFERENCE_OBJECT.relative_direction\",\n",
    "                                   \"location.SPEAKER_LOOK_REL.relative_direction\",\n",
    "                                   \"location.SPEAKER_POS_REL.relative_direction\",\n",
    "                                   \"location.AGENT_POS_REL.relative_direction\"\n",
    "                                   ]\n",
    "        if any(x in reference_location_keys for x in d.keys()):\n",
    "            for k, v in d.items():\n",
    "                if k in reference_location_keys:\n",
    "                    r[\"location\"][\"relative_direction\"] = d.get(k)\n",
    "                    d[k] = None\n",
    "        \n",
    "        # fix steps\n",
    "        if (k.startswith(\"location.REFERENCE_OBJECT.steps\") for k, v in d.items()):\n",
    "            new_d = {}\n",
    "            for k, v in d.items():\n",
    "                if k.startswith(\"location.REFERENCE_OBJECT.steps\"):\n",
    "                    parts = k.split(\".\")\n",
    "                    new_l = [parts[0]]\n",
    "                    new_l.extend(parts[2:])\n",
    "                    new_key = \".\".join(new_l)\n",
    "                    new_d[new_key] = v\n",
    "                else:\n",
    "                    new_d[k ] = v\n",
    "\n",
    "            d = new_d\n",
    "        \n",
    "        if r['location']['location_type'] in ['AGENT_POS_REL', 'SPEAKER_POS_REL', 'SPEAKER_LOOK_REL']:\n",
    "            r['location']['location_type'] = ''.join(r['location']['location_type'][0:-4]) \n",
    "        \n",
    "        if r['location']['location_type'] == 'CONTAINS_COREFERENCE':\n",
    "            del r['location']['location_type']\n",
    "            r['location']['contains_coreference'] = 'yes'\n",
    "            r[\"location\"].update(process_dict(with_prefix(d, \"location.\")))\n",
    "        elif r['location']['location_type'] == 'coordinates_check':\n",
    "            r['location']['location_type'] = 'COORDINATES'\n",
    "            r[\"location\"].update(process_dict(with_prefix(d, \"location.\")))\n",
    "        elif r['location']['location_type'] == 'coref_resolve_check':\n",
    "            del r['location']['location_type']\n",
    "            r[\"location\"].update(process_dict(with_prefix(d, \"location.\")))\n",
    "        elif r[\"location\"][\"location_type\"] == \"REFERENCE_OBJECT\":       \n",
    "            # here just get span of reference object\n",
    "            r[\"location\"][\"location_type\"] = \"REFERENCE_OBJECT\"\n",
    "            # update steps in old data\n",
    "            \n",
    "            if (\"relative_direction\" in r[\"location\"]):\n",
    "                x = process_dict(with_prefix(d, \"location.REFERENCE_OBJECT.relative_direction.{}.\".format(r[\"location\"][\"relative_direction\"])))\n",
    "                #print(x)\n",
    "                r[\"location\"].update(x)\n",
    "                #print(r)\n",
    "                dirn = r[\"location\"][\"relative_direction\"]\n",
    "                #print(dirn)\n",
    "                for k, v in d.items():\n",
    "                    if k.startswith('location.REFERENCE_OBJECT.relative_direction.{}.reference_object.has_name.'.format(dirn)):\n",
    "                        d[k] = None\n",
    "                    if k.startswith('location.REFERENCE_OBJECT.relative_direction.{}.reference_object.location.'.format(dirn)):\n",
    "                        d[k] = None\n",
    "                    if k.startswith(\"location.REFERENCE_OBJECT.relative_direction.{}.reference_object.contains_coreference\".format(dirn)):\n",
    "                        d[k] = None\n",
    "                    if k.startswith('location.REFERENCE_OBJECT.relative_direction.{}.reference_object_1.has_name.'.format(r[\"location\"][\"relative_direction\"])):\n",
    "                        d[k] = None\n",
    "                    if k.startswith(\"location.REFERENCE_OBJECT.relative_direction.{}.reference_object_1.contains_coreference\".format(r[\"location\"][\"relative_direction\"])):\n",
    "                        d[k] = None\n",
    "                    if k.startswith('location.REFERENCE_OBJECT.relative_direction.{}.reference_object_2.has_name.'.format(r[\"location\"][\"relative_direction\"])):\n",
    "                        d[k] = None\n",
    "                    if k.startswith(\"location.REFERENCE_OBJECT.relative_direction.{}.reference_object_2.contains_coreference\".format(r[\"location\"][\"relative_direction\"])):\n",
    "                        d[k] = None\n",
    "            else:\n",
    "                del r[\"location\"][\"location_type\"]\n",
    "            # no key for EXACT\n",
    "        if (\"relative_direction\" in r[\"location\"]) and (r[\"location\"][\"relative_direction\"] in (\"EXACT\", \"Other\")):\n",
    "            del r[\"location\"][\"relative_direction\"]\n",
    "            \n",
    "\n",
    "    \n",
    "    for k, v in d.items():\n",
    "        \n",
    "        if (\n",
    "            k == \"location\"\n",
    "            or k in ['COPY']\n",
    "            or (k == \"relative_direction\" and v in (\"EXACT\", \"NEAR\", \"Other\"))\n",
    "        ):\n",
    "            continue\n",
    "        # handle span\n",
    "        if re.match(\"[^.]+.span#[0-9]+\", k):\n",
    "            prefix, rest = k.split(\".\", 1)\n",
    "            idx = int(rest.split(\"#\")[-1])\n",
    "            # print('here')\n",
    "            if prefix in r:\n",
    "                r[prefix].append([idx, idx])\n",
    "                r[prefix] = sorted(r[prefix], key=itemgetter(0))\n",
    "            else:\n",
    "                r[prefix] = [[idx, idx]]\n",
    "                #r[prefix] = [idx, idx]\n",
    "        elif k == 'reference_object' and v == 'contains_coreference.yes':\n",
    "            r['reference_object'] = {'contains_coreference': 'yes'}\n",
    "\n",
    "        # handle nested dict\n",
    "        elif \".\" in k:\n",
    "            prefix, rest = k.split(\".\", 1)\n",
    "            prefix_snake = snake_case(prefix)\n",
    "            r[prefix_snake] = r.get(prefix_snake, {})\n",
    "            r[prefix_snake].update(process_dict(with_prefix(d, prefix + \".\")))\n",
    "\n",
    "        # handle const value\n",
    "        else:\n",
    "            r[k] = v\n",
    "        \n",
    "    #print(r)\n",
    "    return r\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "   \n",
    "\n",
    "def handle_components(d, child_name):\n",
    "    output = {}\n",
    "    # only handle reference_object\n",
    "    if child_name == 'filters':\n",
    "        output['filters'] = {'reference_object': {}}\n",
    "        if any(k.startswith('reference_object') and v == 'contains_coreference.yes' for k, v in d.items()):\n",
    "            output['filters']['reference_object']['contains_coreference'] = 'yes'\n",
    "\n",
    "        child_d = process_dict(with_prefix(d, \"{}.\".format('reference_object')))\n",
    "        output['filters']['reference_object'].update(child_d)\n",
    "    # ref object with coreference\n",
    "    elif child_name =='reference_object' and any(k.startswith('reference_object') and v == 'contains_coreference.yes' for k, v in d.items()):\n",
    "            output[child_name] = {'contains_coreference': 'yes'}\n",
    "            child_d = process_dict(with_prefix(d, \"{}.\".format(child_name)))\n",
    "            output[child_name].update(child_d)\n",
    "    elif child_name =='reference_object' and any(x in ['special_reference.SPEAKER', 'special_reference.AGENT'] for x in d.values()):\n",
    "            if 'special_reference.SPEAKER' in d.values():\n",
    "                output[child_name] = {'special_reference': 'SPEAKER'}\n",
    "            elif 'special_reference.AGENT' in d.values():\n",
    "                output[child_name] = {'special_reference': 'AGENT'}\n",
    "            child_d = process_dict(with_prefix(d, \"{}.\".format(child_name)))\n",
    "            output[child_name].update(child_d)\n",
    "    else:\n",
    "#         print(child_name)\n",
    "#         print(with_prefix(d, \"{}.\".format(child_name)))\n",
    "        child_d = process_dict(with_prefix(d, \"{}.\".format(child_name)))\n",
    "        \n",
    "        if 'location' in child_d and 'location_type' in child_d['location']:\n",
    "            value = child_d['location']['location_type']  \n",
    "            child_d['location'].pop('location_type')\n",
    "            if value in ['SPEAKER_LOOK', 'AGENT_POS', 'SPEAKER_POS', 'COORDINATES']:\n",
    "                updated_value = value # same for coordinates and speaker_look\n",
    "                if value == 'AGENT_POS':\n",
    "                    updated_value = 'AGENT'\n",
    "                elif value == 'SPEAKER_POS':\n",
    "                    updated_value = 'SPEAKER'\n",
    "                elif value == 'COORDINATES':\n",
    "                    if 'coordinates' in child_d['location']:\n",
    "                        updated_value = {'coordinates_span' : child_d['location']['coordinates']}\n",
    "                    else:\n",
    "                        updated_value = None\n",
    "\n",
    "                # add to reference object instead\n",
    "                if updated_value == None:\n",
    "                    del child_d['location']\n",
    "                else:\n",
    "                    if 'reference_object' in child_d['location']:\n",
    "                        child_d['location']['reference_object']['special_reference'] = updated_value\n",
    "                    else:\n",
    "                        child_d['location']['reference_object'] = {'special_reference' : updated_value}\n",
    "\n",
    "                    if 'coordinates' in child_d['location']:\n",
    "                        del child_d['location']['coordinates']\n",
    "        \n",
    "        output[child_name] = {}\n",
    "        for k, v in child_d.items():\n",
    "            flag = 'yes'\n",
    "            if k == \"comparison\":\n",
    "                flag = 'no'\n",
    "            output[child_name][k] = [flag, child_d[k]] \n",
    "    \n",
    "    \n",
    "    return output\n",
    "\n",
    "def process_result(full_d):\n",
    "    worker_id = full_d[\"WorkerId\"]\n",
    "    action_name = full_d['Input.intent']\n",
    "    #print(action_name)\n",
    "    child_name = full_d['Input.child']\n",
    "    #print(full_d)\n",
    "    d = with_prefix(full_d, \"Answer.root.\"+ action_name + \"!\" + child_name + \".\") # replace with \"Answer.root.\"\n",
    "    #print(d, child_name)\n",
    "    d = fix_nums_at_end_of_ref_obj(d, ['reference_object'])\n",
    "    #print(d)\n",
    "    receiver_flag = False\n",
    "    original_child_name = child_name\n",
    "    if child_name in ['receiver_reference_object', 'source_reference_object']:\n",
    "        child_name = 'reference_object'\n",
    "        receiver_flag = True\n",
    "    action_dict = handle_components(d, child_name)\n",
    "    if receiver_flag:\n",
    "        action_dict[original_child_name] = action_dict[child_name]\n",
    "        action_dict.pop(child_name)\n",
    "    # Fix empty words messing up spans\n",
    "    words = [full_d[\"Input.word{}\".format(x)] for x in range(MAX_WORDS)]\n",
    "    action_dict, words = fix_spans_due_to_empty_words(action_dict, words)\n",
    "\n",
    "    return worker_id, action_dict, words, original_child_name\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "and add <span style='background-color: #FFFF00'>it</span> to the tree\n",
      "{'reference_object': {'contains_coreference': 'yes'}}\n",
      "********************\n",
      "bot bring me <span style='background-color: #FFFF00'>my red pants suit</span>\n",
      "{'reference_object': {'has_colour': ['yes', [[4, 4]]],\n",
      "                      'has_name': ['yes', [[5, 5], [6, 6]]]}}\n",
      "********************\n",
      "bot bring <span style='background-color: #FFFF00'>me</span> my red pants suit\n",
      "{'receiver_reference_object': {'special_reference': 'SPEAKER'}}\n",
      "********************\n",
      "chop a <span style='background-color: #FFFF00'>tree</span>\n",
      "{'reference_object': {'has_name': ['yes', [[2, 2]]]}}\n",
      "********************\n",
      "collect the <span style='background-color: #FFFF00'>mushrooms</span> here\n",
      "{'reference_object': {'has_name': ['yes', [[2, 2]]]}}\n",
      "********************\n",
      "find a <span style='background-color: #FFFF00'>villiage</span>\n",
      "{'reference_object': {'has_name': ['yes', [[2, 2]]]}}\n",
      "********************\n",
      "find <span style='background-color: #FFFF00'>diamons</span>\n",
      "{'reference_object': {'has_name': ['yes', [[1, 1]]]}}\n",
      "********************\n",
      "find me a <span style='background-color: #FFFF00'>cow</span>\n",
      "{'reference_object': {'has_name': ['yes', [[3, 3]]]}}\n",
      "********************\n",
      "get <span style='background-color: #FFFF00'>bricks</span>\n",
      "{'reference_object': {'has_name': ['yes', [[1, 1]]]}}\n",
      "********************\n",
      "get <span style='background-color: #FFFF00'>plant</span>\n",
      "{'reference_object': {'has_name': ['yes', [[1, 1]]]}}\n",
      "********************\n",
      "get <span style='background-color: #FFFF00'>redstone</span> from your inventory\n",
      "{'reference_object': {'has_name': ['yes', [[1, 1]]]}}\n",
      "********************\n",
      "get <span style='background-color: #FFFF00'>wood</span>\n",
      "{'reference_object': {'has_name': ['yes', [[1, 1]]]}}\n",
      "********************\n",
      "get <span style='background-color: #FFFF00'>wool</span>\n",
      "{'reference_object': {'has_name': ['yes', [[1, 1]]]}}\n",
      "********************\n",
      "give me a <span style='background-color: #FFFF00'>axe</span>\n",
      "{'reference_object': {'has_name': ['yes', [[3, 3]]]}}\n",
      "********************\n",
      "give <span style='background-color: #FFFF00'>me</span> a axe\n",
      "{'receiver_reference_object': {'special_reference': 'SPEAKER'}}\n",
      "********************\n",
      "give me a <span style='background-color: #FFFF00'>brick</span>\n",
      "{'reference_object': {'has_name': ['yes', [[3, 3]]]}}\n",
      "********************\n",
      "give <span style='background-color: #FFFF00'>me</span> a brick\n",
      "{'receiver_reference_object': {'special_reference': 'SPEAKER'}}\n",
      "********************\n",
      "give me <span style='background-color: #FFFF00'>stone</span>\n",
      "{'reference_object': {'has_name': ['yes', [[2, 2]]]}}\n",
      "********************\n",
      "give <span style='background-color: #FFFF00'>me</span> stone\n",
      "{'receiver_reference_object': {'special_reference': 'SPEAKER'}}\n",
      "********************\n",
      "give me <span style='background-color: #FFFF00'>wool</span>\n",
      "{'reference_object': {'has_name': ['yes', [[2, 2]]]}}\n",
      "********************\n",
      "give <span style='background-color: #FFFF00'>me</span> wool\n",
      "{'receiver_reference_object': {'special_reference': 'SPEAKER'}}\n",
      "********************\n",
      "go get <span style='background-color: #FFFF00'>flower</span>\n",
      "{'reference_object': {'has_name': ['yes', [[2, 2]]]}}\n",
      "********************\n",
      "hand me <span style='background-color: #FFFF00'>that axe</span>\n",
      "{'reference_object': {'contains_coreference': 'yes', 'has_name': [[3, 3]]}}\n",
      "********************\n",
      "hand <span style='background-color: #FFFF00'>me</span> that axe\n",
      "{'receiver_reference_object': {'special_reference': 'SPEAKER'}}\n",
      "********************\n",
      "hey bot can you grab some <span style='background-color: #FFFF00'>wood</span> for me\n",
      "{'reference_object': {'has_name': ['yes', [[6, 6]]]}}\n",
      "********************\n",
      "hey bot can you grab some wood for <span style='background-color: #FFFF00'>me</span>\n",
      "{'receiver_reference_object': {'special_reference': 'SPEAKER'}}\n",
      "********************\n",
      "move the <span style='background-color: #FFFF00'>book</span> to inside the house\n",
      "{'reference_object': {'has_name': ['yes', [[2, 2]]]}}\n",
      "********************\n",
      "pick <span style='background-color: #FFFF00'>flower</span>\n",
      "{'reference_object': {'has_name': ['yes', [[1, 1]]]}}\n",
      "********************\n",
      "pick <span style='background-color: #FFFF00'>this</span> up\n",
      "{'reference_object': {'contains_coreference': 'yes'}}\n",
      "********************\n",
      "place <span style='background-color: #FFFF00'>block</span>\n",
      "{'reference_object': {'has_name': ['yes', [[1, 1]]]}}\n",
      "********************\n",
      "place five <span style='background-color: #FFFF00'>pigs</span>\n",
      "{'reference_object': {'has_name': ['yes', [[2, 2]]]}}\n",
      "********************\n",
      "place <span style='background-color: #FFFF00'>it</span>\n",
      "{'reference_object': {'contains_coreference': 'yes'}}\n",
      "********************\n",
      "show <span style='background-color: #FFFF00'>fire</span>\n",
      "{'reference_object': {'has_name': ['yes', [[1, 1]]]}}\n",
      "********************\n",
      "take some <span style='background-color: #FFFF00'>dirt</span> from the pile\n",
      "{'reference_object': {'has_name': ['yes', [[2, 2]]]}}\n",
      "********************\n",
      "34\n"
     ]
    }
   ],
   "source": [
    "# convert csv to txt\n",
    "\n",
    "from pprint import pprint\n",
    "import json\n",
    "\n",
    "'''\n",
    "command: Input.command\n",
    "\n",
    "'''\n",
    "from pprint import pprint\n",
    "result_dict = {}\n",
    "folder_name = '/Users/kavyasrinet/Desktop/other_actions/5/toolC/'\n",
    "f_name = folder_name + 'out.csv'\n",
    "\n",
    "only_show_disagreements=True\n",
    "sentence_mapping = {}\n",
    "with open(f_name, \"r\") as f:\n",
    "    r = csv.DictReader(f)\n",
    "    for i, d in enumerate(r):\n",
    "        # print(i, sentence)\n",
    "        #if i==0:\n",
    "        sentence = d['Input.command']\n",
    "        ''' the sentence has a span in it'''\n",
    "        worker_id, action_dict, words, child_name = process_result(d)\n",
    "\n",
    "        print(sentence)\n",
    "        pprint(action_dict)\n",
    "        print(\"*\"*20)\n",
    "\n",
    "        if action_dict is None:\n",
    "            continue\n",
    "        command = \" \".join(words)\n",
    "#         if command == 'bring arthur yoghurt from the fridge':\n",
    "#             print(sentence)\n",
    "#             pprint(action_dict)\n",
    "#             print(\"*\"*20)\n",
    "        command = command + \"$$$\" + child_name\n",
    "\n",
    "        #if command == \"build a statue four feet to your left$$$location\":\n",
    "        sentence_mapping[command] = sentence\n",
    "        result = json.dumps(action_dict)\n",
    "        \n",
    "        if command in result_dict:\n",
    "            if len(result_dict[command]) == 3:\n",
    "                continue\n",
    "            result_dict[command].append(result)\n",
    "        else:\n",
    "            result_dict[command] = [result]\n",
    "print(len(result_dict.keys()))\n",
    "#pprint(result_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write to txt\n",
    "import json\n",
    "f_name = folder_name + 'out.txt'\n",
    "with open(f_name, 'w') as outfile:\n",
    "    for k, v in result_dict.items():\n",
    "        cmd, child  = k.split(\"$$$\")\n",
    "        # len(v) is number of annotations for a command\n",
    "        \n",
    "        if len(v) != 3:\n",
    "            items = v[0] + \"\\t\" + v[0] + \"\\t\" + v[0]\n",
    "#             print(k, v, len(v))\n",
    "#             break\n",
    "        else:\n",
    "            items=  \"\\t\".join(v)\n",
    "        outfile.write(cmd + \"\\t\" + child + \"\\t\" + items+\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_definite_articles(cmd, d):\n",
    "    words = cmd.split()\n",
    "    if type(d) == str:\n",
    "        d = ast.literal_eval(d)\n",
    "    new_d = {}\n",
    "    # print(d)\n",
    "    for k, v in d.items():\n",
    "        # for level 1 \n",
    "        if type(v) == list and v[0] in ['yes', 'no']:\n",
    "            if type(v[1]) == list:\n",
    "                new_v = []\n",
    "                for span in v[1]:\n",
    "                    # span[0] and span[1] are the same\n",
    "                    if words[span[0]] in ['the', 'a', 'an']:\n",
    "                        continue\n",
    "                    new_v.append(span)\n",
    "                new_d[k] = [v[0], new_v]  \n",
    "            elif type(v[1]) == dict:\n",
    "                v_new = remove_definite_articles(cmd, v[1])\n",
    "                new_d[k] = [v[0], v_new]\n",
    "\n",
    "            else:\n",
    "                new_d[k] = v\n",
    "        # for recursion on normal internal dict\n",
    "        else:\n",
    "            if type(v) == list:\n",
    "                new_v = []\n",
    "                for span in v:\n",
    "                    # span[0] and span[1] are the same\n",
    "                    if words[span[0]] in ['the', 'a', 'an']:\n",
    "                        continue\n",
    "                    new_v.append(span)\n",
    "                new_d[k] = new_v  \n",
    "            elif type(v) == dict:\n",
    "                v_new = remove_definite_articles(cmd, v)\n",
    "                new_d[k] = v_new\n",
    "\n",
    "            else:\n",
    "                new_d[k] = v\n",
    "\n",
    "    return new_d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34\n"
     ]
    }
   ],
   "source": [
    "# construct counter from txt\n",
    "result_counts = defaultdict(Counter)\n",
    "import ast\n",
    "f_name = folder_name + 'out.txt'\n",
    "with open(f_name) as in_data:\n",
    "    for line in in_data.readlines():\n",
    "        line = line.strip()\n",
    "        cmd, child, r1, r2, r3 = line.split(\"\\t\")\n",
    "        #print(cmd, child)\n",
    "        for r in [r1, r2, r3]:\n",
    "            r_new = remove_definite_articles(cmd, r)\n",
    "            result_counts[cmd+\"$$$\"+child][json.dumps(r_new)] += 1\n",
    "print(len(result_counts.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "# compute agreements and disagreements\n",
    "no_agreement = 0\n",
    "num_agreements = 2\n",
    "agreement = 0\n",
    "only_show_disagreements = False\n",
    "disagreement = defaultdict(Counter)\n",
    "all_agreements_dict = {}\n",
    "for command, counts in sorted(result_counts.items()):\n",
    "    if not any(v >= num_agreements for v in counts.values()):\n",
    "        if only_show_disagreements:\n",
    "            print(command)\n",
    "        disagreement[command] = counts\n",
    "        no_agreement += 1\n",
    "        continue\n",
    "    elif only_show_disagreements:\n",
    "        continue\n",
    "\n",
    "    #print(command)\n",
    "\n",
    "    for result, count in counts.items():\n",
    "        if count >= num_agreements:\n",
    "            all_agreements_dict[command] = result\n",
    "            agreement += 1\n",
    "            #print(result)\n",
    "\n",
    "    #print()\n",
    "print(agreement)\n",
    "print(no_agreement)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/kavyasrinet/Desktop/other_actions/5/toolC/\n"
     ]
    }
   ],
   "source": [
    "# write out agreements to a file\n",
    "## format is : command child dict\n",
    "print(folder_name)\n",
    "ag = str(agreement)\n",
    "f = folder_name + ag +'_agreements.txt'\n",
    "with open(f, 'w') as outfile:\n",
    "    for k, v in all_agreements_dict.items():\n",
    "        cmd, child = k.split('$$$')\n",
    "        outfile.write(cmd + \"\\t\" + child + \"\\t\" + v + \"\\n\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write disagreements to a file\n",
    "disag = str(no_agreement)\n",
    "f = folder_name + disag + '_disagreements.txt'\n",
    "with open(f, 'w') as outfile:\n",
    "    for k, v in disagreement.items():\n",
    "        cmd, child = k.split('$$$')\n",
    "        outfile.write(cmd+\"\\t\"+child+\"\\n\")\n",
    "        for item in v:\n",
    "            outfile.write(item+\"\\n\")\n",
    "        outfile.write(\"\\n\")\n",
    "        outfile.write(\"\\n\")\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ast\n",
    "from operator import itemgetter\n",
    "\n",
    "from pprint import pprint\n",
    "\n",
    "import ast\n",
    "from operator import itemgetter\n",
    "\n",
    "def resolve_spans(words, dicts):\n",
    "    result = {}\n",
    "    for d, val in dicts.items():\n",
    "        new_d = {}\n",
    "        d = ast.literal_eval(d)\n",
    "        for k1, v1 in d.items():\n",
    "            inner = {}\n",
    "            for k, v in v1.items():\n",
    "                #print(v)\n",
    "                if type(v) == list:\n",
    "                    new_v = []\n",
    "                    for item in v:\n",
    "                        if item[0] == item[1]:\n",
    "                            new_v.append(words[item[0]])\n",
    "#                         else:\n",
    "#                             for item in v:\n",
    "#                                 new_v.append(words[item])\n",
    "                    inner[k] = new_v\n",
    "                elif k =='repeat':\n",
    "                    \n",
    "                    if 'stop_condition' in v:\n",
    "                        new_v = {}\n",
    "                        new_v['stop_condition'] = {}\n",
    "                        x = {}\n",
    "                        x['condition_type'] = v['stop_condition']['condition_type']\n",
    "\n",
    "                        new_vals = []\n",
    "                        if v['stop_condition']['block_type'][0] ==v['stop_condition']['block_type'][1]:\n",
    "                            new_vals.append(words[v['stop_condition']['block_type'][0]])\n",
    "                        else:\n",
    "                            for item in v['stop_condition']['block_type']:\n",
    "                                new_vals.append(words[item])\n",
    "                        x['block_type'] = new_vals\n",
    "                        new_v['stop_condition'] = x\n",
    "                        inner['repeat'] = new_v \n",
    "                else:\n",
    "                    inner[k] = v\n",
    "            new_d[k1] = inner\n",
    "        result[str(new_d)] = val\n",
    "    return result\n",
    "\n",
    "for command, counts in disagreement.items():\n",
    "    words = command.split()\n",
    "    parts = words[-1].split(\"$$$\")\n",
    "    print(sentence_mapping[command])\n",
    "    words[-1] = parts[0]\n",
    "    child_name = parts[1]\n",
    "    command = \" \".join(words)\n",
    "    #print(words, counts)\n",
    "    c = resolve_spans(words, counts)\n",
    "    print(command, child_name)\n",
    "    #print(counts)\n",
    "    # print(\"*\"*30)\n",
    "    for k, v in c.items():\n",
    "        pprint(ast.literal_eval(k))\n",
    "        print(\"-\"*10)\n",
    "    #print(c)\n",
    "    print(\"*\"*30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(folder_name+'/all_agreements.txt', 'w') as f, \\\n",
    "     open(folder_name + ag +'_agreements.txt') as f1, \\\n",
    "     open(folder_name + disag + '_disagreements.txt') as f2:\n",
    "    for line in f1.readlines():\n",
    "        line = line.strip()\n",
    "        f.write(line+ \"\\n\")\n",
    "    for line in f2.readlines():\n",
    "        line = line.strip()\n",
    "        f.write(line+ \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
